using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

namespace LiveStock.Recommendation
{
  class HttpTextGrabber
  {
    /**
     * This function obtains the body text of a web page, minus the
     * HTML.
     */
    public static string GrabTextFromUrl(string url)
    {
      StringBuilder sb = new StringBuilder();
      string result = null;
      byte[] buf = new byte[10240];

      // Get HTML from Internet
      try
      {
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
        HttpWebResponse response = (HttpWebResponse)request.GetResponse();
        Stream stream = response.GetResponseStream();

        string tmp = null;
        int count = 0;

        do
        {
          count = stream.Read(buf, 0, buf.Length);
          if (count > 0)
          {
            tmp = Encoding.ASCII.GetString(buf, 0, count);
            sb.Append(tmp);
          }
        }
        while (count > 0);
      }
      catch
      {
        // Ignore error.
      }

      // Strip out HTML tags between BODY tags.
      if (sb.Length > 0)
      {
        int start = sb.ToString().ToUpper().IndexOf("<BODY");
        start = sb.ToString().IndexOf(">", start) + 1;

        int stop = sb.ToString().ToUpper().IndexOf("</BODY");

        if (start == -1) start = 0;
        if (stop == -1) stop = sb.Length;

        if (start < stop)
        {
          StringBuilder plaintext = new StringBuilder();
          string content = StripScripts(sb.ToString(start, stop - start));
          content = StripDoubleLineEndings(content);

          for (Match match = Regex.Match(content, @"([^<]*)<[^>]*>"); match.Success; match = match.NextMatch())
          {
            GroupCollection gc = match.Groups;
            plaintext.Append(gc[1].Value);
          }

          result = plaintext.ToString();
        }
      }

      return result;
    }


    /**
     * Removes text between SCRIPT tags which could appear within the
     * body of a web page.
     */
    public static string StripScripts(string str)
    {
      string result = null;
      int start = 0, stop = 0;

      if (str != null && str.Length > 0)
      {
        string uppercased = str.ToUpper();
        StringBuilder sb = new StringBuilder();

        while ((stop = uppercased.IndexOf("<SCRIPT", start)) >= 0)
        {
          sb.Append(str.Substring(start, stop - start));
          start = uppercased.IndexOf("</SCRIPT>", start) + "</SCRIPT>".Length;
        }

        if (start < str.Length) sb.Append(str.Substring(start));

        result = sb.ToString();
      }

      return result;
    }


    /**
     * Removes chains of 'carriage returns' and 'new lines'.
     */
    private static string StripDoubleLineEndings(string str)
    {
      string result = str;

      if (result != null && result.Length > 0)
      {
        result = result.Replace("\r\n\r\n", "\r\n");
        result = result.Replace("\n\n", "\n");
        result = result.Replace("\r\r", "\r");
      }

      return result;
    }

  }
}
